{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8dfe2ce9-9cf9-4559-8b0b-64e363c3c564",
   "metadata": {},
   "outputs": [],
   "source": [
    "mr_data = {\n",
    "    \"mrrel\":{\n",
    "        \"cols\":['CUI1', 'AUI1', 'STYPE1', 'REL', 'CUI2', 'AUI2', 'STYPE2', 'RELA', 'RUI', 'SRUI', 'SAB', 'SL', 'RG', 'DIR', 'SUPPRESS', 'CVF'],\n",
    "        \"not-required-cols\": ['STYPE1', 'SL', 'RG', 'DIR', 'SUPPRESS', 'CVF'],\n",
    "        \"path\": \"../../datasets/TaskA/UMLS/raw/umls-2022AB-metathesaurus-full/2022AB/META/MRREL.RRF\"\n",
    "    },\n",
    "    \"mrconso\":{\n",
    "        \"cols\":['CUI', 'LAT', 'TS', 'LUI', 'STT', 'SUI', 'ISPREF', 'AUI', 'SAUI', 'SCUI', 'SDUI', 'SAB', \n",
    "                'TTY', 'CODE', 'STR', 'SRL', 'SUPPRESS', 'CVF'],\n",
    "        \"not-required-cols\": ['TS', 'LUI', 'STT', 'SUI', 'ISPREF','TTY', 'SCUI', 'SDUI', 'CODE', 'SRL', 'SUPPRESS', 'CVF'],\n",
    "        \"path\": \"../../datasets/TaskA/UMLS/raw/umls-2022AB-metathesaurus-full/2022AB/META/MRCONSO.RRF\"\n",
    "    },\n",
    "    \"mrsty\":{\n",
    "        \"cols\":['CUI', 'TUI', 'STN', 'STY','ATUI', 'CVF'],\n",
    "        \"not-required-cols\": ['CVF'],\n",
    "        \"path\": \"../../datasets/TaskA/UMLS/raw/umls-2022AB-metathesaurus-full/2022AB/META/MRSTY.RRF\"\n",
    "    },\n",
    "    \"mrsat\":{\n",
    "        \"cols\":['CUI', 'LUI', 'SUI', 'METAUI', 'STYPE', 'CODE','ATUI', 'SATUI', 'ATN', 'SAB', 'ATV', 'SUPPRESS','CVF'],\n",
    "        \"not-required-cols\": ['SUPPRESS', 'CVF'],\n",
    "        \"path\": \"../../datasets/TaskA/UMLS/raw/umls-2022AB-metathesaurus-full/2022AB/META/MRSAT.RRF\"\n",
    "    }\n",
    "    \n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "505610b9-b2cb-4e98-ba94-b18f8aafb6cd",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Defnitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5ad53ee6-4646-4865-9cc6-76c1678a14b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "def load_print_columns(path, columns_name, not_required_cols, demo):\n",
    "    df_data_dict = []\n",
    "    false_line = 0\n",
    "    with open(path, 'r') as file_obj:\n",
    "        for index_line, line in tqdm(enumerate(file_obj)):\n",
    "            requireds, line_status = {}, True\n",
    "            \n",
    "            for index_column, line_item in enumerate(line.split(\"|\")[:-1]):\n",
    "                if columns_name[index_column] not in not_required_cols:\n",
    "                    requireds[columns_name[index_column]] = line_item\n",
    "                    # if line_item == \"\":\n",
    "                    #     line_status = False\n",
    "            if line_status:    \n",
    "                if demo:\n",
    "                    df_data_dict.append(list(requireds.values()))\n",
    "                    if index_line == 10:\n",
    "                        df = pd.DataFrame(df_data_dict, columns=[column_name \n",
    "                                                                 for column_name in columns_name \n",
    "                                                                 if column_name not in not_required_cols])\n",
    "                        display(df.head(4))\n",
    "                        return df\n",
    "\n",
    "                else:\n",
    "                    yield requireds\n",
    "            else:\n",
    "                false_line += 1\n",
    "    print(\"False lines:\", false_line)\n",
    "\n",
    "def get_generator(data):\n",
    "    dataset = load_print_columns(mr_data[data]['path'], \n",
    "                             mr_data[data]['cols'], \n",
    "                             mr_data[data]['not-required-cols'], \n",
    "                             demo=False)\n",
    "    return dataset\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def make_bar_plot(X, Y, figsize, title):\n",
    "    fig = plt.figure(figsize=figsize)\n",
    "    ax = fig.add_axes([0,0,1,1])\n",
    "    ax.bar(X, Y)\n",
    "    ax.set_title(title)\n",
    "    plt.xticks(rotation=90)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "931822a7-df9f-43fe-8d06-e65ec4890af4",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1. First Glance on MRREL, MRCONSO, and MRSTY files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a402fb7-c3a3-4e9d-a2b1-753caf6a8a1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WORKING‌‌ ON‌‌:: MRREL\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38f2376e79fe43e2918fe18ade894792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CUI1</th>\n",
       "      <th>AUI1</th>\n",
       "      <th>REL</th>\n",
       "      <th>CUI2</th>\n",
       "      <th>AUI2</th>\n",
       "      <th>STYPE2</th>\n",
       "      <th>RELA</th>\n",
       "      <th>RUI</th>\n",
       "      <th>SRUI</th>\n",
       "      <th>SAB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>A13433185</td>\n",
       "      <td>RB</td>\n",
       "      <td>C0036775</td>\n",
       "      <td>A7466261</td>\n",
       "      <td>SCUI</td>\n",
       "      <td></td>\n",
       "      <td>R86000559</td>\n",
       "      <td></td>\n",
       "      <td>MSHFRE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>A26634265</td>\n",
       "      <td>RB</td>\n",
       "      <td>C0036775</td>\n",
       "      <td>A0115649</td>\n",
       "      <td>SCUI</td>\n",
       "      <td></td>\n",
       "      <td>R31979041</td>\n",
       "      <td></td>\n",
       "      <td>MSH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>A0016515</td>\n",
       "      <td>SY</td>\n",
       "      <td>C0000039</td>\n",
       "      <td>A11754881</td>\n",
       "      <td>AUI</td>\n",
       "      <td>translation_of</td>\n",
       "      <td>R101808683</td>\n",
       "      <td></td>\n",
       "      <td>MSHSWE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>A0016515</td>\n",
       "      <td>SY</td>\n",
       "      <td>C0000039</td>\n",
       "      <td>A12080359</td>\n",
       "      <td>AUI</td>\n",
       "      <td>sort_version_of</td>\n",
       "      <td>R64565540</td>\n",
       "      <td></td>\n",
       "      <td>MSH</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       CUI1       AUI1 REL      CUI2       AUI2 STYPE2             RELA  \\\n",
       "0  C0000005  A13433185  RB  C0036775   A7466261   SCUI                    \n",
       "1  C0000005  A26634265  RB  C0036775   A0115649   SCUI                    \n",
       "2  C0000039   A0016515  SY  C0000039  A11754881    AUI   translation_of   \n",
       "3  C0000039   A0016515  SY  C0000039  A12080359    AUI  sort_version_of   \n",
       "\n",
       "          RUI SRUI     SAB  \n",
       "0   R86000559       MSHFRE  \n",
       "1   R31979041          MSH  \n",
       "2  R101808683       MSHSWE  \n",
       "3   R64565540          MSH  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b3d303de6dc4997a5f6842ac5853c5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'AUI1': 'A13433185',\n",
      " 'AUI2': 'A7466261',\n",
      " 'CUI1': 'C0000005',\n",
      " 'CUI2': 'C0036775',\n",
      " 'REL': 'RB',\n",
      " 'RELA': '',\n",
      " 'RUI': 'R86000559',\n",
      " 'SAB': 'MSHFRE',\n",
      " 'SRUI': '',\n",
      " 'STYPE2': 'SCUI'}\n",
      "----------------------------------------------------------------------------------------------------\n",
      "WORKING‌‌ ON‌‌:: MRCONSO\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6a8ad62096b4eef868eb64249ac9ddd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CUI</th>\n",
       "      <th>LAT</th>\n",
       "      <th>AUI</th>\n",
       "      <th>SAUI</th>\n",
       "      <th>SAB</th>\n",
       "      <th>STR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>ENG</td>\n",
       "      <td>A26634265</td>\n",
       "      <td></td>\n",
       "      <td>MSH</td>\n",
       "      <td>(131)I-Macroaggregated Albumin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>ENG</td>\n",
       "      <td>A26634266</td>\n",
       "      <td></td>\n",
       "      <td>MSH</td>\n",
       "      <td>(131)I-MAA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>FRE</td>\n",
       "      <td>A13433185</td>\n",
       "      <td></td>\n",
       "      <td>MSHFRE</td>\n",
       "      <td>Macroagrégats d'albumine marquée à l'iode 131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>FRE</td>\n",
       "      <td>A27488794</td>\n",
       "      <td></td>\n",
       "      <td>MSHFRE</td>\n",
       "      <td>MAA-I 131</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        CUI  LAT        AUI SAUI     SAB  \\\n",
       "0  C0000005  ENG  A26634265          MSH   \n",
       "1  C0000005  ENG  A26634266          MSH   \n",
       "2  C0000005  FRE  A13433185       MSHFRE   \n",
       "3  C0000005  FRE  A27488794       MSHFRE   \n",
       "\n",
       "                                             STR  \n",
       "0                 (131)I-Macroaggregated Albumin  \n",
       "1                                     (131)I-MAA  \n",
       "2  Macroagrégats d'albumine marquée à l'iode 131  \n",
       "3                                      MAA-I 131  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffe523d932404a388b63c6cb8bff1790",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'AUI': 'A26634265',\n",
      " 'CUI': 'C0000005',\n",
      " 'LAT': 'ENG',\n",
      " 'SAB': 'MSH',\n",
      " 'SAUI': '',\n",
      " 'STR': '(131)I-Macroaggregated Albumin'}\n",
      "----------------------------------------------------------------------------------------------------\n",
      "WORKING‌‌ ON‌‌:: MRSTY\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5994b92b0d154f83ad9591666b51f5f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CUI</th>\n",
       "      <th>TUI</th>\n",
       "      <th>STN</th>\n",
       "      <th>STY</th>\n",
       "      <th>ATUI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>T116</td>\n",
       "      <td>A1.4.1.2.1.7</td>\n",
       "      <td>Amino Acid, Peptide, or Protein</td>\n",
       "      <td>AT17648347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>T121</td>\n",
       "      <td>A1.4.1.1.1</td>\n",
       "      <td>Pharmacologic Substance</td>\n",
       "      <td>AT17575038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>T130</td>\n",
       "      <td>A1.4.1.1.4</td>\n",
       "      <td>Indicator, Reagent, or Diagnostic Aid</td>\n",
       "      <td>AT17634323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>T109</td>\n",
       "      <td>A1.4.1.2.1</td>\n",
       "      <td>Organic Chemical</td>\n",
       "      <td>AT45562015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        CUI   TUI           STN                                    STY  \\\n",
       "0  C0000005  T116  A1.4.1.2.1.7        Amino Acid, Peptide, or Protein   \n",
       "1  C0000005  T121    A1.4.1.1.1                Pharmacologic Substance   \n",
       "2  C0000005  T130    A1.4.1.1.4  Indicator, Reagent, or Diagnostic Aid   \n",
       "3  C0000039  T109    A1.4.1.2.1                       Organic Chemical   \n",
       "\n",
       "         ATUI  \n",
       "0  AT17648347  \n",
       "1  AT17575038  \n",
       "2  AT17634323  \n",
       "3  AT45562015  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50871198a2a14009ade14a63748988ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ATUI': 'AT17648347',\n",
      " 'CUI': 'C0000005',\n",
      " 'STN': 'A1.4.1.2.1.7',\n",
      " 'STY': 'Amino Acid, Peptide, or Protein',\n",
      " 'TUI': 'T116'}\n",
      "----------------------------------------------------------------------------------------------------\n",
      "WORKING‌‌ ON‌‌:: MRSAT\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3287ec847ea5425c9d897c6cb1a84758",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CUI</th>\n",
       "      <th>LUI</th>\n",
       "      <th>SUI</th>\n",
       "      <th>METAUI</th>\n",
       "      <th>STYPE</th>\n",
       "      <th>CODE</th>\n",
       "      <th>ATUI</th>\n",
       "      <th>SATUI</th>\n",
       "      <th>ATN</th>\n",
       "      <th>SAB</th>\n",
       "      <th>ATV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>L0000005</td>\n",
       "      <td>S0007492</td>\n",
       "      <td>A26634265</td>\n",
       "      <td>AUI</td>\n",
       "      <td>D012711</td>\n",
       "      <td>AT212456753</td>\n",
       "      <td></td>\n",
       "      <td>TH</td>\n",
       "      <td>MSH</td>\n",
       "      <td>UNK (19XX)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>L0000005</td>\n",
       "      <td>S0007492</td>\n",
       "      <td>A26634265</td>\n",
       "      <td>AUI</td>\n",
       "      <td>D012711</td>\n",
       "      <td>AT212480766</td>\n",
       "      <td></td>\n",
       "      <td>TERMUI</td>\n",
       "      <td>MSH</td>\n",
       "      <td>T037573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>L0000005</td>\n",
       "      <td>S0007492</td>\n",
       "      <td>A26634265</td>\n",
       "      <td>SCUI</td>\n",
       "      <td>D012711</td>\n",
       "      <td>AT60774257</td>\n",
       "      <td></td>\n",
       "      <td>RN</td>\n",
       "      <td>MSH</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000005</td>\n",
       "      <td>L0270109</td>\n",
       "      <td>S0007491</td>\n",
       "      <td>A26634266</td>\n",
       "      <td>AUI</td>\n",
       "      <td>D012711</td>\n",
       "      <td>AT212327137</td>\n",
       "      <td></td>\n",
       "      <td>TERMUI</td>\n",
       "      <td>MSH</td>\n",
       "      <td>T037574</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        CUI       LUI       SUI     METAUI STYPE     CODE         ATUI SATUI  \\\n",
       "0  C0000005  L0000005  S0007492  A26634265   AUI  D012711  AT212456753         \n",
       "1  C0000005  L0000005  S0007492  A26634265   AUI  D012711  AT212480766         \n",
       "2  C0000005  L0000005  S0007492  A26634265  SCUI  D012711   AT60774257         \n",
       "3  C0000005  L0270109  S0007491  A26634266   AUI  D012711  AT212327137         \n",
       "\n",
       "      ATN  SAB         ATV  \n",
       "0      TH  MSH  UNK (19XX)  \n",
       "1  TERMUI  MSH     T037573  \n",
       "2      RN  MSH           0  \n",
       "3  TERMUI  MSH     T037574  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd4266ccea0943249dd1e5a9005675fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ATN': 'TH',\n",
      " 'ATUI': 'AT212456753',\n",
      " 'ATV': 'UNK (19XX)',\n",
      " 'CODE': 'D012711',\n",
      " 'CUI': 'C0000005',\n",
      " 'LUI': 'L0000005',\n",
      " 'METAUI': 'A26634265',\n",
      " 'SAB': 'MSH',\n",
      " 'SATUI': '',\n",
      " 'STYPE': 'AUI',\n",
      " 'SUI': 'S0007492'}\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for data in mr_data.keys():\n",
    "    print(f\"WORKING‌‌ ON‌‌:: {data.upper()}\")\n",
    "    \n",
    "    print(*load_print_columns(mr_data[data]['path'], \n",
    "                              mr_data[data]['cols'], \n",
    "                              mr_data[data]['not-required-cols'], \n",
    "                              demo=True))\n",
    "    \n",
    "    pprint(next(load_print_columns(mr_data[data]['path'], \n",
    "                                  mr_data[data]['cols'], \n",
    "                                  mr_data[data]['not-required-cols'], \n",
    "                                  demo=False)))\n",
    "    \n",
    "    print(\"-\"*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0538073-9eef-47c0-993b-931f2fa1e599",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 2. create UMLS_feb_skiped_bad_lines.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "29a4d30e-1787-46e4-bfd3-e350074553c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "sabs_to_consider = ['NCI', 'SNOMEDCT_US', 'MEDCIN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f6b32f7d-c03b-45e6-9bb3-3390f0283e01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting SAB and STRs from MRCONSO-ENG...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df01800eef3f4c4285aeee2aa80ecf01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6db871e7b06c466f8ce9bb0f7d2395fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False lines: 0\n"
     ]
    }
   ],
   "source": [
    "print(\"Getting SAB and STRs from MRCONSO-ENG...\")\n",
    "sab_names = {}\n",
    "for conso in tqdm(get_generator(data = 'MRCONSO'.lower())):\n",
    "    if conso['LAT'] == 'ENG' and conso['SAB'] in sabs_to_consider:\n",
    "        sab_names[conso['CUI']+\"-\"+conso['AUI']] = f\"{conso['SAB']}@{conso['STR']}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "362ec052-12bc-42e7-b1f8-5a830f38e39d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('C0000039-A22817493', 'SNOMEDCT_US@Dipalmitoylphosphatidylcholine')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(sab_names.keys())[0], sab_names[list(sab_names.keys())[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a1304f81-1090-4af7-acd4-918011b8881c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting CONSO-REL rels and ents based on intersections....\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1ae9fc8699049ea83ed4e4eb0102337",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9aea0899a1e14471bde3a75de128d4c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False lines: 0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"Getting CONSO-REL rels and ents based on intersections....\")\n",
    "conso_rel_intersection_rels_ent = defaultdict(int)\n",
    "dataset = get_generator(data = 'MRREL'.lower())\n",
    "sep = \"\\t\"\n",
    "for sample in tqdm(dataset):\n",
    "    found = {}\n",
    "    head, tail = False, False\n",
    "    if sab_names.get(sample['CUI1']+\"-\"+sample['AUI1'], \"NA\") != \"NA\":\n",
    "        found['head'] = f\"{sep}\".join(sab_names[sample['CUI1']+\"-\"+sample['AUI1']].split(\"@\"))\n",
    "        head = True\n",
    "    if sab_names.get(sample['CUI2']+\"-\"+sample['AUI2'], \"NA\") != \"NA\":\n",
    "        found['tail'] = f\"{sep}\".join(sab_names[sample['CUI2']+\"-\"+sample['AUI2']].split(\"@\"))\n",
    "        tail = True \n",
    "    if head and tail:\n",
    "        # file.write(f\"{sample['CUI1']}{sep}{sample['CUI2']}{sep}{sample['RELA']}{sep}{found['head']}{sep}{found['tail']}\\n\")\n",
    "        conso_rel_intersection_rels_ent[f\"{sample['CUI1']}{sep}{sample['AUI1']}{sep}{sample['RUI']}{sep}{sample['CUI2']}{sep}{sample['AUI2']}{sep}{sample['RELA']}{sep}{found['head']}{sep}{found['tail']}\"] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9f101758-82cc-49d7-809b-693b17551ddf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique samples that avaliable in both MRCONSO-ENG and MRREL:11455094\n",
      "Number of apperances of samples in both MRCONSO-ENG and MRREL:11455094\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b3bf05d929412e82f369a4e118bbd3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11455094 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(f\"Number of unique samples that avaliable in both MRCONSO-ENG and MRREL:{len(conso_rel_intersection_rels_ent)}\")\n",
    "print(f\"Number of apperances of samples in both MRCONSO-ENG and MRREL:{sum(conso_rel_intersection_rels_ent.values())}\")\n",
    "\n",
    "with open(\"../../datasets/TaskA/UMLS/processed-3/UMLS_feb.tsv\", \"a\") as file:\n",
    "    for line in tqdm(conso_rel_intersection_rels_ent.keys()):\n",
    "        file.write(str(line)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2675f9cc-5a7e-4081-a6fd-a5f3ea32b6bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Previous shape of UMLS initial dataset was: 11455094\n",
      "New size of UMLS is: 11454956\n",
      "Bad lines that skiped:‌ 138\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"../../datasets/TaskA/UMLS/processed-3/UMLS_feb.tsv\", sep=\"\\t\", on_bad_lines='skip',\n",
    "                 names=[\"CUI1\", \"AUI1\", \"RUI\", \"CUI2\", \"AUI2\", \"RELA\", \"SAB-CUI1\", \"STR-CUI1\", \"SAB-CUI2\", \"STR-CUI2\"])\n",
    "\n",
    "print(f\"Previous shape of UMLS initial dataset was: 11455094\")\n",
    "print(f\"New size of UMLS is: {df.shape[0]}\")\n",
    "print(f\"Bad lines that skiped:‌ {11455094-df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "25f53749-4f80-41b4-9f90-99b0d01b3ae2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"../../datasets/TaskA/UMLS/processed-3/UMLS_feb_skiped_bad_lines.tsv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0b73ccc-e9fd-47ed-b946-4cf8f3c42471",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3. Generate Ontos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0345b5c4-fa9c-439d-9f38-d3918531ceeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'ATUI': 'AT17648347',\n",
    "#  'CUI': 'C0000005',\n",
    "#  'STN': 'A1.4.1.2.1.7',\n",
    "#  'STY': 'Amino Acid, Peptide, or Protein',\n",
    "#  'TUI': 'T116'}\n",
    "\n",
    "df = pd.read_csv(\"../../datasets/TaskA/UMLS/processed-3/UMLS_feb_skiped_bad_lines.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "31641f1d-3b7b-4dfa-bae0-af5caca1f77b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CUI1</th>\n",
       "      <th>AUI1</th>\n",
       "      <th>RUI</th>\n",
       "      <th>CUI2</th>\n",
       "      <th>AUI2</th>\n",
       "      <th>RELA</th>\n",
       "      <th>SAB-CUI1</th>\n",
       "      <th>STR-CUI1</th>\n",
       "      <th>SAB-CUI2</th>\n",
       "      <th>STR-CUI2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>A22817493</td>\n",
       "      <td>R20277339</td>\n",
       "      <td>C0031610</td>\n",
       "      <td>A2941532</td>\n",
       "      <td>inverse_isa</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Dipalmitoylphosphatidylcholine</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Phosphatidic acid</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>A22817493</td>\n",
       "      <td>R13958460</td>\n",
       "      <td>C0523614</td>\n",
       "      <td>A3120482</td>\n",
       "      <td>has_component</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Dipalmitoylphosphatidylcholine</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Dipalmitoylphosphatidylcholine measurement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>A22817493</td>\n",
       "      <td>R141817308</td>\n",
       "      <td>C0523614</td>\n",
       "      <td>A3120482</td>\n",
       "      <td>has_measured_component</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Dipalmitoylphosphatidylcholine</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Dipalmitoylphosphatidylcholine measurement</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000039</td>\n",
       "      <td>A22817493</td>\n",
       "      <td>R142021020</td>\n",
       "      <td>C0216971</td>\n",
       "      <td>A3008866</td>\n",
       "      <td>same_as</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Dipalmitoylphosphatidylcholine</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Colfosceril palmitate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000052</td>\n",
       "      <td>A27769867</td>\n",
       "      <td>R141668442</td>\n",
       "      <td>C0019495</td>\n",
       "      <td>A3493340</td>\n",
       "      <td>inverse_isa</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>1,4-alpha-Glucan branching enzyme</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>Hexosyltransferase</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       CUI1       AUI1         RUI      CUI2      AUI2  \\\n",
       "0  C0000039  A22817493   R20277339  C0031610  A2941532   \n",
       "1  C0000039  A22817493   R13958460  C0523614  A3120482   \n",
       "2  C0000039  A22817493  R141817308  C0523614  A3120482   \n",
       "3  C0000039  A22817493  R142021020  C0216971  A3008866   \n",
       "4  C0000052  A27769867  R141668442  C0019495  A3493340   \n",
       "\n",
       "                     RELA     SAB-CUI1                           STR-CUI1  \\\n",
       "0             inverse_isa  SNOMEDCT_US     Dipalmitoylphosphatidylcholine   \n",
       "1           has_component  SNOMEDCT_US     Dipalmitoylphosphatidylcholine   \n",
       "2  has_measured_component  SNOMEDCT_US     Dipalmitoylphosphatidylcholine   \n",
       "3                 same_as  SNOMEDCT_US     Dipalmitoylphosphatidylcholine   \n",
       "4             inverse_isa  SNOMEDCT_US  1,4-alpha-Glucan branching enzyme   \n",
       "\n",
       "      SAB-CUI2                                    STR-CUI2  \n",
       "0  SNOMEDCT_US                           Phosphatidic acid  \n",
       "1  SNOMEDCT_US  Dipalmitoylphosphatidylcholine measurement  \n",
       "2  SNOMEDCT_US  Dipalmitoylphosphatidylcholine measurement  \n",
       "3  SNOMEDCT_US                       Colfosceril palmitate  \n",
       "4  SNOMEDCT_US                          Hexosyltransferase  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "195b0b48-e573-489c-a965-7f8742897b84",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 3.1 NCI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "bde6841f-766e-453a-a6ff-74a39106a66a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07abb356425547f4b55edcb0b9db0217",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e63848f05f4c4ed78c88fbb7f2261556",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False lines: 0\n",
      "\n",
      "Out of 172538 Unique CUIs for 'NCI' we found Entity types for 172538's\n",
      "\n",
      "A few samples from unique_cuis: ['C0796615', 'C0298456', 'C4527089', 'C0458924', 'C0855054']\n",
      "\n",
      "Examples: C0728966: [['T121'], ['A1.4.1.1.1'], ['Pharmacologic Substance']]\n"
     ]
    }
   ],
   "source": [
    "# ['NCI', 'SNOMEDCT_US', 'MEDCIN']\n",
    "sab_to_consider = 'NCI'\n",
    "\n",
    "sab_rel_df = df[df['SAB-CUI1'].isin([sab_to_consider]) & df['SAB-CUI2'].isin([sab_to_consider])].reset_index()\n",
    "sab_ents_list = sab_rel_df['CUI1'].tolist() + sab_rel_df['CUI2'].tolist()\n",
    "sab_ents_dict = {sab:\"OK\" for sab in list(set(sab_ents_list))}\n",
    "\n",
    "entity_type_dict = {}\n",
    "\n",
    "for mrsty in tqdm(get_generator(data = 'MRSTY'.lower())):\n",
    "    if sab_ents_dict.get(mrsty['CUI'], \"NA\") != \"NA\":\n",
    "        if mrsty['CUI'] in entity_type_dict.get(mrsty['CUI'], \"NA\") != \"NA\":\n",
    "            entity_type_dict[mrsty['CUI']][0].append(mrsty['TUI'])\n",
    "            entity_type_dict[mrsty['CUI']][1].append(mrsty['STN'])\n",
    "            entity_type_dict[mrsty['CUI']][2].append(mrsty['STY'])\n",
    "        else:\n",
    "            entity_type_dict[mrsty['CUI']] = [[mrsty['TUI']], [mrsty['STN']], [mrsty['STY']]]\n",
    "\n",
    "print(f\"\\nOut of {len(sab_ents_dict)} Unique CUIs for '{sab_to_consider}' we found Entity types for {len(entity_type_dict)}'s\")\n",
    "\n",
    "print(\"\\nA few samples from unique_cuis:\", list(sab_ents_dict.keys())[:5])\n",
    "\n",
    "print(f\"\\nExamples: {list(sab_ents_dict.keys())[10]}: {entity_type_dict[list(sab_ents_dict.keys())[10]]}\")\n",
    "\n",
    "\n",
    "stn2str, tui2str = {}, {}\n",
    "tui2stn, tuis, stns = {}, [], []\n",
    "\n",
    "for cui, items in entity_type_dict.items():\n",
    "    tui, stn, string = items[0], items[1], items[2]\n",
    "    for item1, item2, item3 in zip(tui, stn, string):\n",
    "        if stn2str.get(item2, \"NA\") != \"NA\":\n",
    "            if stn2str.get(item2) != item3:\n",
    "                print(f\"conflict-stn2str: {item1}, {item2}, {item3} and ==> {stn2str[item2]}\")\n",
    "        else:\n",
    "            stn2str[item2] = item3\n",
    "            \n",
    "        if tui2str.get(item1, \"NA\") != \"NA\":\n",
    "            if tui2str.get(item1) != item3:\n",
    "                print(f\"conflict-tui2str: {item1}, {item2}, {item3} and ==> {tui2str[item1]}\")\n",
    "        else:\n",
    "            tui2str[item1] = item3 \n",
    "            \n",
    "        if tui2stn.get(item1, \"NA\") != \"NA\":\n",
    "            if tui2stn.get(item1) != item2:\n",
    "                print(f\"conflict-tui2stn: {item1}, {item2}, {item3} and ==> {tui2stn[item1]}\")\n",
    "        else:\n",
    "            tui2stn[item1] = item2 \n",
    "        \n",
    "        \n",
    "        \n",
    "        tuis.append(item1)\n",
    "        stns.append(item2)\n",
    "\n",
    "assert len(stn2str) == len(tui2str)\n",
    "assert len(tui2str) == len(tui2stn)\n",
    "assert len(tui2str) == len(set(tuis))\n",
    "assert len(set(tuis)) == len(set(stns))\n",
    "\n",
    "final_json = {\n",
    "    \"SAB\": sab_to_consider,\n",
    "    \"STN2STR\": stn2str,\n",
    "    \"TUI2STR\": tui2str,\n",
    "    \"TUIs\": list(set(tuis)),\n",
    "    \"STNs\": list(set(stns)),\n",
    "    \"TUI2STN\": tui2stn\n",
    "}\n",
    "\n",
    "with open(sab_to_consider+\"_hierarchy.json\", \"w\", encoding=\"utf-8\") as outfile:\n",
    "     json.dump(final_json, outfile, indent=4, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b98ffb7-8946-4304-99a7-6f5886b93f10",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 3.2 MEDCIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9ebbf3cd-f71a-4224-b600-00d212416b0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1091748dca74ed09a5f00d402e4de46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f107b67ea08740dd87025dbe39457b55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False lines: 0\n",
      "\n",
      "Out of 377587 Unique CUIs for 'MEDCIN' we found Entity types for 377587's\n",
      "\n",
      "A few samples from unique_cuis: ['C2087929', 'C2139321', 'C2182110', 'C2012112', 'C2077859']\n",
      "\n",
      "Examples: C2123362: [['T034'], ['A2.2.1'], ['Laboratory or Test Result']]\n"
     ]
    }
   ],
   "source": [
    "# ['NCI', 'SNOMEDCT_US', 'MEDCIN']\n",
    "sab_to_consider = 'MEDCIN'\n",
    "\n",
    "sab_rel_df = df[df['SAB-CUI1'].isin([sab_to_consider]) & df['SAB-CUI2'].isin([sab_to_consider])].reset_index()\n",
    "sab_ents_list = sab_rel_df['CUI1'].tolist() + sab_rel_df['CUI2'].tolist()\n",
    "sab_ents_dict = {sab:\"OK\" for sab in list(set(sab_ents_list))}\n",
    "\n",
    "entity_type_dict = {}\n",
    "\n",
    "for mrsty in tqdm(get_generator(data = 'MRSTY'.lower())):\n",
    "    if sab_ents_dict.get(mrsty['CUI'], \"NA\") != \"NA\":\n",
    "        if mrsty['CUI'] in entity_type_dict.get(mrsty['CUI'], \"NA\") != \"NA\":\n",
    "            entity_type_dict[mrsty['CUI']][0].append(mrsty['TUI'])\n",
    "            entity_type_dict[mrsty['CUI']][1].append(mrsty['STN'])\n",
    "            entity_type_dict[mrsty['CUI']][2].append(mrsty['STY'])\n",
    "        else:\n",
    "            entity_type_dict[mrsty['CUI']] = [[mrsty['TUI']], [mrsty['STN']], [mrsty['STY']]]\n",
    "\n",
    "print(f\"\\nOut of {len(sab_ents_dict)} Unique CUIs for '{sab_to_consider}' we found Entity types for {len(entity_type_dict)}'s\")\n",
    "\n",
    "print(\"\\nA few samples from unique_cuis:\", list(sab_ents_dict.keys())[:5])\n",
    "\n",
    "print(f\"\\nExamples: {list(sab_ents_dict.keys())[10]}: {entity_type_dict[list(sab_ents_dict.keys())[10]]}\")\n",
    "\n",
    "\n",
    "stn2str, tui2str = {}, {}\n",
    "tui2stn, tuis, stns = {}, [], []\n",
    "\n",
    "for cui, items in entity_type_dict.items():\n",
    "    tui, stn, string = items[0], items[1], items[2]\n",
    "    for item1, item2, item3 in zip(tui, stn, string):\n",
    "        if stn2str.get(item2, \"NA\") != \"NA\":\n",
    "            if stn2str.get(item2) != item3:\n",
    "                print(f\"conflict-stn2str: {item1}, {item2}, {item3} and ==> {stn2str[item2]}\")\n",
    "        else:\n",
    "            stn2str[item2] = item3\n",
    "            \n",
    "        if tui2str.get(item1, \"NA\") != \"NA\":\n",
    "            if tui2str.get(item1) != item3:\n",
    "                print(f\"conflict-tui2str: {item1}, {item2}, {item3} and ==> {tui2str[item1]}\")\n",
    "        else:\n",
    "            tui2str[item1] = item3 \n",
    "            \n",
    "        if tui2stn.get(item1, \"NA\") != \"NA\":\n",
    "            if tui2stn.get(item1) != item2:\n",
    "                print(f\"conflict-tui2stn: {item1}, {item2}, {item3} and ==> {tui2stn[item1]}\")\n",
    "        else:\n",
    "            tui2stn[item1] = item2 \n",
    "        \n",
    "        \n",
    "        \n",
    "        tuis.append(item1)\n",
    "        stns.append(item2)\n",
    "\n",
    "assert len(stn2str) == len(tui2str)\n",
    "assert len(tui2str) == len(tui2stn)\n",
    "assert len(tui2str) == len(set(tuis))\n",
    "assert len(set(tuis)) == len(set(stns))\n",
    "\n",
    "final_json = {\n",
    "    \"SAB\": sab_to_consider,\n",
    "    \"STN2STR\": stn2str,\n",
    "    \"TUI2STR\": tui2str,\n",
    "    \"TUIs\": list(set(tuis)),\n",
    "    \"STNs\": list(set(stns)),\n",
    "    \"TUI2STN\": tui2stn\n",
    "}\n",
    "\n",
    "with open(sab_to_consider+\"_hierarchy.json\", \"w\", encoding=\"utf-8\") as outfile:\n",
    "     json.dump(final_json, outfile, indent=4, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff9f45f1-a816-4677-9d68-6d5043de8fde",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## 3.3 SNOMEDCT_US"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a5497001-3db3-4e73-8ccd-9e3045ded438",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a759ca3d15a542ee9c90ea1b083a6f93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b43ecd02bfa4a569b587b3d40c688cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False lines: 0\n",
      "\n",
      "Out of 417172 Unique CUIs for 'SNOMEDCT_US' we found Entity types for 417172's\n",
      "\n",
      "A few samples from unique_cuis: ['C2711921', 'C0228062', 'C0230393', 'C1276816', 'C3204456']\n",
      "\n",
      "Examples: C2960408: [['T060'], ['B1.3.1.2'], ['Diagnostic Procedure']]\n"
     ]
    }
   ],
   "source": [
    "# ['NCI', 'SNOMEDCT_US', 'MEDCIN']\n",
    "sab_to_consider = 'SNOMEDCT_US'\n",
    "\n",
    "sab_rel_df = df[df['SAB-CUI1'].isin([sab_to_consider]) & df['SAB-CUI2'].isin([sab_to_consider])].reset_index()\n",
    "sab_ents_list = sab_rel_df['CUI1'].tolist() + sab_rel_df['CUI2'].tolist()\n",
    "sab_ents_dict = {sab:\"OK\" for sab in list(set(sab_ents_list))}\n",
    "\n",
    "entity_type_dict = {}\n",
    "\n",
    "for mrsty in tqdm(get_generator(data = 'MRSTY'.lower())):\n",
    "    if sab_ents_dict.get(mrsty['CUI'], \"NA\") != \"NA\":\n",
    "        if mrsty['CUI'] in entity_type_dict.get(mrsty['CUI'], \"NA\") != \"NA\":\n",
    "            entity_type_dict[mrsty['CUI']][0].append(mrsty['TUI'])\n",
    "            entity_type_dict[mrsty['CUI']][1].append(mrsty['STN'])\n",
    "            entity_type_dict[mrsty['CUI']][2].append(mrsty['STY'])\n",
    "        else:\n",
    "            entity_type_dict[mrsty['CUI']] = [[mrsty['TUI']], [mrsty['STN']], [mrsty['STY']]]\n",
    "\n",
    "print(f\"\\nOut of {len(sab_ents_dict)} Unique CUIs for '{sab_to_consider}' we found Entity types for {len(entity_type_dict)}'s\")\n",
    "\n",
    "print(\"\\nA few samples from unique_cuis:\", list(sab_ents_dict.keys())[:5])\n",
    "\n",
    "print(f\"\\nExamples: {list(sab_ents_dict.keys())[10]}: {entity_type_dict[list(sab_ents_dict.keys())[10]]}\")\n",
    "\n",
    "\n",
    "stn2str, tui2str = {}, {}\n",
    "tui2stn, tuis, stns = {}, [], []\n",
    "\n",
    "for cui, items in entity_type_dict.items():\n",
    "    tui, stn, string = items[0], items[1], items[2]\n",
    "    for item1, item2, item3 in zip(tui, stn, string):\n",
    "        if stn2str.get(item2, \"NA\") != \"NA\":\n",
    "            if stn2str.get(item2) != item3:\n",
    "                print(f\"conflict-stn2str: {item1}, {item2}, {item3} and ==> {stn2str[item2]}\")\n",
    "        else:\n",
    "            stn2str[item2] = item3\n",
    "            \n",
    "        if tui2str.get(item1, \"NA\") != \"NA\":\n",
    "            if tui2str.get(item1) != item3:\n",
    "                print(f\"conflict-tui2str: {item1}, {item2}, {item3} and ==> {tui2str[item1]}\")\n",
    "        else:\n",
    "            tui2str[item1] = item3 \n",
    "            \n",
    "        if tui2stn.get(item1, \"NA\") != \"NA\":\n",
    "            if tui2stn.get(item1) != item2:\n",
    "                print(f\"conflict-tui2stn: {item1}, {item2}, {item3} and ==> {tui2stn[item1]}\")\n",
    "        else:\n",
    "            tui2stn[item1] = item2 \n",
    "        \n",
    "        \n",
    "        \n",
    "        tuis.append(item1)\n",
    "        stns.append(item2)\n",
    "\n",
    "assert len(stn2str) == len(tui2str)\n",
    "assert len(tui2str) == len(tui2stn)\n",
    "assert len(tui2str) == len(set(tuis))\n",
    "assert len(set(tuis)) == len(set(stns))\n",
    "\n",
    "final_json = {\n",
    "    \"SAB\": sab_to_consider,\n",
    "    \"STN2STR\": stn2str,\n",
    "    \"TUI2STR\": tui2str,\n",
    "    \"TUIs\": list(set(tuis)),\n",
    "    \"STNs\": list(set(stns)),\n",
    "    \"TUI2STN\": tui2stn\n",
    "}\n",
    "\n",
    "with open(sab_to_consider+\"_hierarchy.json\", \"w\", encoding=\"utf-8\") as outfile:\n",
    "     json.dump(final_json, outfile, indent=4, ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e33e5af3-06f9-48b8-bf69-b5786b775d29",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
